{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This is the script used to determine the random examples to prune in the benchmarking figure (dotted red line) in the paper. This has no algorihtmic purpose but is included for reproducibility."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from torchvision import datasets\n",
    "from sklearn.metrics import accuracy_score\n",
    "import cleanlab\n",
    "from IPython.display import Image, display\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# urllib2 for python2 and python3\n",
    "try:\n",
    "    # For Python 3.0 and later\n",
    "    from urllib.request import urlopen\n",
    "except ImportError:\n",
    "    # Fall back to Python 2's urllib2\n",
    "    from urllib2 import urlopen\n",
    "    \n",
    "# simple label names for ImageNet\n",
    "url = 'https://raw.githubusercontent.com/anishathalye/imagenet-simple-labels/master/imagenet-simple-labels.json'\n",
    "f = urlopen(url)\n",
    "simple_labels = json.loads('\\n'.join(i.decode('ascii') for i in f.readlines()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load psx, labels, and image locations\n",
    "data_dir = \"/home/cgn/datasets/datasets/imagenet/\"\n",
    "traindir = data_dir + \"train/\"\n",
    "# psx = np.load(data_dir + \"imagenet__train__model_resnet50__pyx.npy\")\n",
    "imgs, labels = [list(z) for  z in zip(*datasets.ImageFolder(traindir).imgs)]\n",
    "labels = np.array(labels, dtype=int)\n",
    "# print('Overall accuracy: {:.2%}'.format(accuracy_score(labels, psx.argmax(axis = 1))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creates seperate files for the top 20% errors, 40% errors,...\n",
    "for i in range(1,5):\n",
    "    # Prepare arguments\n",
    "    amt = str(100 * i // 5)\n",
    "    end_idx = len(label_errors_idx) * i // 5\n",
    "    partial_errors_idx = label_errors_idx[:end_idx]\n",
    "    # Create new bool mask\n",
    "    bool_mask = np.zeros(len(label_errors_bool), dtype=bool)\n",
    "    bool_mask[partial_errors_idx] = True\n",
    "    # Validate\n",
    "    assert(all(np.array([i for i, b in enumerate(bool_mask) if b]) == sorted(partial_errors_idx)))\n",
    "    print(amt, end_idx)\n",
    "    np.save(\"imagenet_train_RANDOM_bool_mask__fraction_{}.npy\".format(amt), ~bool_mask)\n",
    "    \n",
    "# Verify written files\n",
    "for i in range(1, 5):\n",
    "    amt = str(100 * i // 5)\n",
    "    end_idx = len(label_errors_idx) * i // 5\n",
    "    truth = np.array(sorted(label_errors_idx[:end_idx]))\n",
    "    us = np.array([i for i, b in enumerate(~np.load(\"imagenet_train_RANDOM_bool_mask__fraction_{}.npy\".format(amt))) if b])\n",
    "    assert(all(truth == us))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Take the opposite of the stored file (errors should be true, not false)\n",
    "label_errors_bool = ~np.load(\"imagenet_train_bool_mask.npy\")\n",
    "num_errors = sum(label_errors_bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1245509\n",
      "1209850\n",
      "1174191\n",
      "1138532\n",
      "1102873\n"
     ]
    }
   ],
   "source": [
    "for frac in [0.2, 0.4, 0.6, 0.8, 1.0]:\n",
    "    mask = np.ones(len(labels), dtype = bool)\n",
    "    selection = np.random.choice(len(labels), int(num_errors * frac), replace = False)\n",
    "    mask[selection] = False\n",
    "    np.save(\"imagenet_train_RANDOM_bool_mask__fraction_{}.npy\".format(frac), mask)\n",
    "    assert(len(label_errors_bool) - int(num_errors * frac) == sum(mask))\n",
    "    print(sum(mask))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "assert(len(label_errors_bool) - int(num_errors * 0.2) == sum(mask))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
